// SPDX-License-Identifier: Apache-2.0
import { filter } from "@thi.ng/transducers/filter";

// TODO split up into groups
export const DEFAULT_STOP_WORDS_EN = new Set([
	"a",
	"above",
	"across",
	"after",
	"against",
	"along",
	"also",
	"among",
	"an",
	"and",
	"any",
	"are",
	"around",
	"at",
	"be",
	"been",
	"before",
	"being",
	"below",
	"beneath",
	"between",
	"beyond",
	"both",
	"but",
	"by",
	"could",
	"despite",
	"do",
	"does",
	"down",
	"during",
	"each",
	"eg",
	"etc",
	"even",
	"every",
	"few",
	"for",
	"from",
	"he",
	"i",
	"ie",
	"in",
	"inside",
	"into",
	"is",
	"it",
	"just",
	"less",
	"many",
	"may",
	"might",
	"more",
	"much",
	"must",
	"of",
	"on",
	"one",
	"onto",
	"or",
	"out",
	"outside",
	"over",
	"over",
	"quite",
	"really",
	"several",
	"she",
	"should",
	"since",
	"so",
	"some",
	"such",
	"that",
	"the",
	"these",
	"they",
	"this",
	"those",
	"three",
	"through",
	"to",
	"too",
	"toward",
	"two",
	"under",
	"until",
	"up",
	"very",
	"was",
	"we",
	"were",
	"with",
	"within",
	"without",
	"would",
	"you",
]);

export const removeStopWords = (words = DEFAULT_STOP_WORDS_EN) =>
	filter((x: string) => !words.has(x));
